This notebook will examine the effectiveness of training a Radial Basis Function (RBF) network for a binary classification task on a two-dimensional dataset with 500 instances.
First we must load the dataset:
In [1]:
library("corpcor")
circle<-read.table("circle.txt",sep=",",header=T)
circle
Next we visualize the dataset:
In [2]:
class.index<-dim(circle)[2]
plot(circle[,]$V1,circle[,]$V2,xlim=c(-1,1),ylim=c(-1,1),col=c("blue","black","red")[circle[,]$labels+2])
Next we perform a 70-30 train/test split:
In [3]:
train.index<-sample(nrow(circle),nrow(circle)*0.3)
train.index
In [4]:
training.set<-circle[train.index,]
training.set
In [5]:
training.set.features<-training.set[,-class.index]
training.set.labels<-training.set[,class.index]
training.set.labels
In [6]:
test.set<-circle[-train.index,]
test.set
In [7]:
test.set.features<-test.set[,-class.index]
test.set.labels<-test.set[,class.index]
test.set.labels
Next we provide a function to train the RBF network and execute the training. We will train an RBF model with 2 centers, 5 centers, and 10 centers, respectively.
In [8]:
rbf <- function(X, Y, K, gamma=1.0) {
N<- dim(X)[1] # number of instances
repeat {
km <- kmeans(X, K) # let's cluster K centers out of the dataset
if (min(km$size)>0) # only accept if there are no empty clusters
break
}
mus <- km$centers # the clusters points
Phi <- matrix(rep(NA,(K+1)*N), ncol=K+1)
for (lin in 1:N) {
Phi[lin,1] <- 1 # bias column
for (col in 1:K) {
Phi[lin,col+1] <- exp( -gamma * norm(as.matrix(X[lin,]-mus[col,]),"F")^2 )
}
}
# w <- pseudoinverse(t(Phi) %*% Phi) %*% t(Phi) %*% matrix(as.numeric(Y)) # find RBF weights
w <- pseudoinverse(Phi) %*% matrix(as.numeric(Y)) # find RBF weights
list(weights=w, centers=mus, gamma=gamma) # return the rbf model
}
# now call rbf function
rbf.model.2<-rbf(training.set.features,training.set.labels, 2)
rbf.model.2
In [9]:
rbf.model.5<-rbf(training.set.features,training.set.labels, 5)
rbf.model.5
In [10]:
rbf.model.10<-rbf(training.set.features,training.set.labels, 10)
rbf.model.10
Next we provide a function to make predictions using our trained models:
In [11]:
rbf.predict <- function(model, X, classification=FALSE) {
gamma <- model$gamma
centers <- model$centers
w <- model$weights
N <- dim(X)[1] # number of observations
pred <- rep(w[1],N) # we need to init to a value, so let's start with the bias
for (j in 1:N) {
# find prediction for point xj
for (k in 1:length(centers[,1])) {
# the weight for center[k] is given by w[k+1] (because w[1] is the bias)
pred[j] <- pred[j] + w[k+1] * exp( -gamma * norm(as.matrix(X[j,]-centers[k,]),"F")^2 )
}
}
if (classification) {
pred <- unlist(lapply(pred, sign))
}
return(pred)
}
predictions.2<-rbf.predict(rbf.model.2,test.set.features,TRUE)
predictions.2
In [12]:
predictions.5<-rbf.predict(rbf.model.5,test.set.features,TRUE)
predictions.5
In [13]:
predictions.10<-rbf.predict(rbf.model.10,test.set.features,TRUE)
predictions.10
Next we present the confusion matrices to validate the predictions:
In [14]:
cm.2<-table(test.set.labels,predictions.2)
cm.2
In [15]:
cm.5<-table(test.set.labels,predictions.5)
cm.5
In [16]:
cm.10<-table(test.set.labels,predictions.10)
cm.10
In [17]:
acc.2<-(cm.2[1][1]+cm.2[4][1])/sum(cm.2)
caption.2<-sprintf("RBF Accuracy: %.4f",acc.2)
caption.2
In [18]:
acc.5<-(cm.5[1][1]+cm.5[4][1])/sum(cm.5)
caption.5<-sprintf("RBF Accuracy: %.4f",acc.5)
caption.5
In [19]:
acc.10<-(cm.10[1][1]+cm.10[4][1])/sum(cm.10)
caption.10<-sprintf("RBF Accuracy: %.4f",acc.10)
caption.10
Next we visualize the predictions with the RBF centers:
In [20]:
plot(training.set[,]$V1,training.set[,]$V2,xlim=c(-1,1),ylim=c(-1,1),col=c("blue","black","red")[training.set.labels[]+2])
points(test.set[,1],test.set[,2],col=c("blue","black","red")[predictions.2[]+2],pch=3)
points(test.set[,1],test.set[,2],col=c("blue","black","red")[test.set.labels[]+2],pch=0)
points(rbf.model.2$centers, col="black", pch=19)
legend("topleft",legend=c("label","prediction"),pch=c(0,3))
title(caption.2)
In [21]:
plot(training.set[,]$V1,training.set[,]$V2,xlim=c(-1,1),ylim=c(-1,1),col=c("blue","black","red")[training.set.labels[]+2])
points(test.set[,1],test.set[,2],col=c("blue","black","red")[predictions.5[]+2],pch=3)
points(test.set[,1],test.set[,2],col=c("blue","black","red")[test.set.labels[]+2],pch=0)
points(rbf.model.5$centers, col="black", pch=19)
legend("topleft",legend=c("label","prediction"),pch=c(0,3))
title(caption.5)
In [22]:
plot(training.set[,]$V1,training.set[,]$V2,xlim=c(-1,1),ylim=c(-1,1),col=c("blue","black","red")[training.set.labels[]+2])
points(test.set[,1],test.set[,2],col=c("blue","black","red")[predictions.10[]+2],pch=3)
points(test.set[,1],test.set[,2],col=c("blue","black","red")[test.set.labels[]+2],pch=0)
points(rbf.model.10$centers, col="black", pch=19)
legend("topleft",legend=c("label","prediction"),pch=c(0,3))
title(caption.10)
Next we compare the performance of an RBF network with 2 centers, 5 centers, and 10 centers to the performance of a neural network with 1 hidden layer containing 2 hidden nodes, 5 hidden nodes, and 10 hidden nodes, respectively. We perform the same process with the neural network as we did with the RBF network:
In [23]:
library(neuralnet)
nrepeats<-5
net.2<-neuralnet(labels ~ V1+V2,training.set,hidden=2,rep=nrepeats)
summary(net.2)
In [24]:
net.5<-neuralnet(labels ~ V1+V2,training.set,hidden=5,rep=nrepeats)
summary(net.5)
In [25]:
net.10<-neuralnet(labels ~ V1+V2,training.set,hidden=10,rep=nrepeats)
summary(net.10)
In [26]:
nn.predictions.2<-predict(net.2,test.set[,-3])
nn.predictions.2<-ifelse(nn.predictions.2>0.5,1,-1)
nn.predictions.2
In [27]:
nn.predictions.5<-predict(net.5,test.set[,-3])
nn.predictions.5<-ifelse(nn.predictions.5>0.5,1,-1)
nn.predictions.5
In [28]:
nn.predictions.10<-predict(net.10,test.set[,-3])
nn.predictions.10<-ifelse(nn.predictions.10>0.5,1,-1)
nn.predictions.10
In [29]:
nn.cm.2<-table(test.set.labels,nn.predictions.2)
nn.cm.2
In [30]:
nn.cm.5<-table(test.set.labels,nn.predictions.5)
nn.cm.5
In [31]:
nn.cm.10<-table(test.set.labels,nn.predictions.10)
nn.cm.10
In [32]:
nn.acc.2<-(nn.cm.2[1][1]+nn.cm.2[4][1])/sum(nn.cm.2)
nn.caption.2<-sprintf("One Hidden Layer (%d nodes) NN Accuracy: %.4f",2,nn.acc.2)
nn.caption.2
In [33]:
nn.acc.5<-(nn.cm.5[1][1]+nn.cm.5[4][1])/sum(nn.cm.5)
nn.caption.5<-sprintf("One Hidden Layer (%d nodes) NN Accuracy: %.4f",5,nn.acc.5)
nn.caption.5
In [34]:
nn.acc.10<-(nn.cm.10[1][1]+nn.cm.10[4][1])/sum(nn.cm.10)
nn.caption.10<-sprintf("One Hidden Layer (%d nodes) NN Accuracy: %.4f",10,nn.acc.10)
nn.caption.10
Finally we compare the performance of the two network types:
In [35]:
rbf.accuracy.vector<-c(acc.2, acc.5, acc.10)
nn.accuracy.vector<-c(nn.acc.2, nn.acc.5, nn.acc.10)
cenhid.vector<-c(2, 5, 10)
accuracy.comp.frame<-as.data.frame(cbind(rbf.accuracy.vector, nn.accuracy.vector, cenhid.vector))
names(accuracy.comp.frame)<-c("RBF Accuracy", "NN Accuracy", "Centers/Hidden Nodes")
accuracy.comp.frame
It is evident from these results that the RBF network performs much better on this datset than a Neural Network with one hidden layer. Also, it took much more time to train the Neural Network than it did the RBF network. The best overall performance was obtained with an RBF network with 5 centers.